{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "#定义神经网络结构\n",
    "class Network(object):\n",
    "    def __init__(self, sizes):\n",
    "        # 网络层数\n",
    "        self.num_layers = len(sizes)\n",
    "        # 每层神经元的个数\n",
    "        self.sizes = sizes\n",
    "        # 初始化每层的偏置\n",
    "        self.biases = [np.random.randn(y, 1) for y in sizes[1:]]\n",
    "        # 初始化每层的权重\n",
    "        self.weights = [np.random.randn(y, x) for x, y in zip(sizes[:-1], sizes[1:])]\n",
    "\n",
    "    def feedforward(self, a):\n",
    "        for b, w in zip(self.biases, self.weights):\n",
    "            a = sigmoid(np.dot(w, a) + b)\n",
    "        return a\n",
    "\n",
    "    # 梯度下降\n",
    "    def SGD(self, training_data, epochs, mini_batch_size, eta, test_data=None):\n",
    "        if test_data:\n",
    "            n_test = len(test_data)\n",
    "        # 训练数据总个数\n",
    "        n = len(training_data)\n",
    "\n",
    "        # 开始训练，循环每一个epochs\n",
    "        for j in range(epochs):  # 在python2.7中为xrange\n",
    "            # 洗牌 打乱训练数据\n",
    "            random.shuffle(training_data)\n",
    "\n",
    "            # mini_batch\n",
    "            mini_batches = [training_data[k:k + mini_batch_size] for k in range(0, n, mini_batch_size)]\n",
    "\n",
    "            # 训练mini_batch\n",
    "            for mini_batch in mini_batches:\n",
    "                self.update_mini_batch(mini_batch, eta)\n",
    "\n",
    "            if test_data:\n",
    "                print(\"Epoch {0}: {1} / {2}\".format(j, self.evaluate(test_data), n_test))\n",
    "            print(\"Epoch {0} complete\".format(j))\n",
    "\n",
    "    def update_mini_batch(self, mini_batch, eta):\n",
    "        # 保存每层偏导\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "\n",
    "        # 训练一个mini_batch\n",
    "        for x, y in mini_batch:\n",
    "            delta_nabla_b, delta_nabla_w = self.update(x, y)\n",
    "\n",
    "            # 保存一次训练网络中每层的偏导\n",
    "            nabla_b = [nb + dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]\n",
    "            nabla_w = [nw + dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]\n",
    "\n",
    "        # 更新权重和偏置 Wn+1 = Wn - eta * nw\n",
    "        self.weights = [w - (eta / len(mini_batch)) * nw for w, nw in zip(self.weights, nabla_w)]\n",
    "        self.biases = [b - (eta / len(mini_batch)) * nb for b, nb in zip(self.biases, nabla_b)]\n",
    "\n",
    "    # 前向传播\n",
    "    def update(self, x, y):\n",
    "        # 保存每层偏导\n",
    "        nabla_b = [np.zeros(b.shape) for b in self.biases]\n",
    "        nabla_w = [np.zeros(w.shape) for w in self.weights]\n",
    "        activation = x  # 保存的输入（训练数据）\n",
    "\n",
    "        # 保存每一层的激励值a=sigmoid(z)\n",
    "        activations = [x]\n",
    "\n",
    "        # 保存每一层的z=wx+b\n",
    "        zs = []\n",
    "        # 前向传播\n",
    "        for b, w in zip(self.biases, self.weights):\n",
    "            # 计算每层的z\n",
    "            z = np.dot(w, activation) + b\n",
    "\n",
    "            # 保存每层的z\n",
    "            zs.append(z)\n",
    "\n",
    "            # 计算每层的a\n",
    "            activation = sigmoid(z)\n",
    "\n",
    "            # 保存每一层的a\n",
    "            activations.append(activation)\n",
    "        # 反向更新\n",
    "        # 计算最后一层的误差\n",
    "        delta = self.cost_derivative(activations[-1], y) * sigmoid_prime(zs[-1])\n",
    "\n",
    "        # 最后一层权重和偏置的导数\n",
    "        nabla_b[-1] = delta\n",
    "        nabla_w[-1] = np.dot(delta, activations[-2].T)\n",
    "\n",
    "        # 倒数第二层一直到第一层 权重和偏置的导数\n",
    "        for l in range(2, self.num_layers):\n",
    "            z = zs[-l]\n",
    "\n",
    "            sp = sigmoid_prime(z)\n",
    "\n",
    "            # 当前层的误差\n",
    "            delta = np.dot(self.weights[-l + 1].T, delta) * sp\n",
    "\n",
    "            # 当前层的偏置和权重的导数\n",
    "            nabla_b[-l] = delta\n",
    "            nabla_w[-l] = np.dot(delta, activations[-l - 1].T)\n",
    "\n",
    "        return (nabla_b, nabla_w)\n",
    "\n",
    "    def evaluate(self, test_data):\n",
    "        test_results = [(np.argmax(self.feedforward(x)), y) for (x, y) in test_data]\n",
    "        return sum(int(x == y) for x, y in test_results)\n",
    "\n",
    "    def cost_derivative(self, output_activation, y):\n",
    "        return (output_activation - y)\n",
    "\n",
    "\n",
    "# sigmoid激励函数\n",
    "def sigmoid(z):\n",
    "    return 1.0 / (1.0 + np.exp(-z))\n",
    "\n",
    "\n",
    "def sigmoid_prime(z):\n",
    "    return sigmoid(z) * (1 - sigmoid(z))\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    from keras.datasets import mnist\n",
    "    from keras.utils import np_utils\n",
    "\n",
    "    # 加载数据\n",
    "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "    x_train = x_train.astype(\"float32\") / 255\n",
    "    x_test = x_test.astype(\"float32\") / 255\n",
    "\n",
    "    y_train = np_utils.to_categorical(y_train, num_classes=10)\n",
    "    y_test = np_utils.to_categorical(y_test, num_classes=10)\n",
    "\n",
    "    # 28*28=784个像素， 可以定义30个神经元， 共有10种分类\n",
    "    net = Network([784, 30, 10])\n",
    "    net.SGD(x_train, 30, 10, 0.5, test_data=y_test)\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
